# 本文件用于执行数据质量检测
# 功能：检测数据质量并计算质量分数
# 返回内容说明（JSON格式）：
#   - duplicate_samples: 重复样本统计
#       - count: 重复样本数量 (int)
#       - ratio: 重复样本占比 (float)
#   - empty_texts: 空文本统计
#       - count: 空文本数量 (int)
#       - ratio: 空文本占比 (float)
#   - abnormal_chars: 异常字符统计
#       - count: 异常字符数量 (int)
#       - ratio: 异常字符占比 (float)
#   - quality_score: 质量分数 (float，满分100)

import json
import re
from collections import Counter
from data_load import load_data,load_stopwords



def get_quality_score(data):
    """执行质量检测并计算质量分数"""
    texts = [item['text'] for item in data.values()]
    stopwords = load_stopwords()

    # 重复样本率
    all_texts = [text for text in texts]
    unique_texts = set(all_texts)
    duplicate_count = len(all_texts) - len(unique_texts)
    duplicate_ratio = round(duplicate_count / len(all_texts), 4)

    # 空文本检测
    empty_count = sum(1 for text in texts if not text.strip())
    empty_ratio = round(empty_count / len(texts), 4)

    # 异常字符检测（非中文/英文/数字/常见符号）
    valid_chars = r'[\u4e00-\u9fa5a-zA-Z0-9，。、；：？！（）【】“”‘’\s]'
    abnormal_count = 0
    for text in texts:
        if re.search(rf'[^{valid_chars}]', text):
            abnormal_count += 1
    abnormal_ratio = round(abnormal_count / len(texts), 4)

    # 质量分数计算（权重：重复50%，空文本30%，异常字符20%）
    quality_score = 100 - (
            duplicate_ratio * 100 +
            empty_ratio * 100 +
            abnormal_ratio * 100
    )
    quality_score = max(0, round(quality_score, 2))

    return {
        "duplicate_samples": {
            "count": duplicate_count,
            "ratio": duplicate_ratio
        },
        "empty_texts": {
            "count": empty_count,
            "ratio": empty_ratio
        },
        "abnormal_chars": {
            "count": abnormal_count,
            "ratio": abnormal_ratio
        },
        "quality_score": quality_score
    }


if __name__ == "__main__":
    # 测试用例
    data = load_data()
    quality = get_quality_score(data)
    print(json.dumps(quality, indent=2, ensure_ascii=False))